﻿' helpful links
' http://johnholliday.net/archive/2006/11/04/Extending-STSADM-with-Custom-Commands.aspx
' http://sharepointsolutions.blogspot.com/2006/09/extending-stsadmexe-with-custom.html
' http://stsadm.blogspot.com/2007/08/stsadm-commands_09.html
' http://andrewconnell.com/blog/articles/MossStsadmWcmCommands.aspx

Imports Microsoft.SharePoint
Imports Microsoft.SharePoint.Administration
Imports Microsoft.SharePoint.Search.Administration
Imports Microsoft.SharePoint.StsAdmin
Imports System.Linq
Imports System.Reflection
Imports System.Data

Public Class ExportCrawlLog : Implements ISPStsadmCommand

    Protected reflected As New Dictionary(Of String, Type)
    Private arguments As New ExportCrawlLogArgs

    Sub New()
        'Referenced assemblies are loaded on demand
        Dim referencedAssemblies As AssemblyName() = Assembly.GetExecutingAssembly().GetReferencedAssemblies()
        Dim sharepointsearchName = referencedAssemblies.Where(Function(x) x.Name = "Microsoft.SharePoint.Search").Single()
        Dim sharepointsearch As Assembly = Assembly.Load(sharepointsearchName)

        'there types are `internal' in Sharepoint foundation 2010
        For Each typename_ In {"Microsoft.SharePoint.Search.Administration.LogViewer",
                               "Microsoft.SharePoint.Search.Administration.CrawlLogSummaryProperty",
                               "Microsoft.SharePoint.Search.Administration.CrawlLogFilters",
                               "Microsoft.SharePoint.Search.Administration.CrawlLogFilterProperty",
                               "Microsoft.SharePoint.Search.Administration.StringFilterOperator",
                               "Microsoft.SharePoint.Search.Administration.MessageType",
                               "Microsoft.SharePoint.Search.Administration.MSSITLB.IContentSource"
                              }
            reflected(typename_.Split("."c).Last()) = sharepointsearch.GetType(typename_, True)
        Next
    End Sub

    Public Function GetHelpMessage(ByVal command As String) As String Implements Microsoft.SharePoint.StsAdmin.ISPStsadmCommand.GetHelpMessage
        Dim hlpMsg As New Text.StringBuilder
        hlpMsg.AppendLine("-t (d|s|c|p) [-site <portal url>] [-outfile <filename>] [-history] [-s startdatetime] [-e enddatetime] [-from #] [-thru #] [-cat (Portal_Content|ProfileImport)] [-csid #] [-msgid #] [-u <url pattern>] [-hostname <hostname>] [-mt (s|w|e)]")
        hlpMsg.AppendLine("  where -t = type: d=details, s=summary, c=content sources and start addresses, p=purgelog")
        hlpMsg.AppendLine("  where -mt = Message type to include in output (applies to -t d only, if -mt is used only one message type can be pecified, when -mt is omitted all types are emitted): s=success, w=warnings, e=errors")
        hlpMsg.AppendLine()
        hlpMsg.AppendLine("Examples:")
        hlpMsg.AppendLine(" stsadm -o ExportCrawlLog -t d -site ""http://moss.litwareinc.com"" -history -outfile output.log -s ""2007-10-30T15:10:26-05:00"" -e ""2007-11-01 11:00"" -mt e")
        hlpMsg.AppendLine(" stsadm -o ExportCrawlLog -t s -site ""http://moss.litwareinc.com"" ")
        hlpMsg.AppendLine(" stsadm -o ExportCrawlLog -t c -site ""http://moss.litwareinc.com"" ")
        Return hlpMsg.ToString()
    End Function

    Public Function Run(ByVal command As String, ByVal keyValues As System.Collections.Specialized.StringDictionary, ByRef output As String) As Integer Implements Microsoft.SharePoint.StsAdmin.ISPStsadmCommand.Run
        If ParseArgs(keyValues) = False Then
            output = GetHelpMessage(command)
            Return -1
        End If
        Dim outTextWriter As System.IO.TextWriter

        If Not arguments.outfile Is Nothing Then
            outTextWriter = New IO.StreamWriter(IO.File.Create(arguments.outfile))
        Else
            outTextWriter = Console.Out
        End If

        'Dim sc As SPServiceContext
        'Using siteCollection As New SPSite(arguments.site)
        '    sc = SPServiceContext.GetContext(siteCollection)
        'End Using

        Dim farm As SPFarm = SPFarm.Local
        Dim svcSearchService As SPSearchService = DirectCast(
            farm.Services.Where(Function(x) TypeOf (x) Is SPSearchService).Single(), 
            SPSearchService)
        Dim siSearchServiceInstance As SPSearchServiceInstance = DirectCast(
            svcSearchService.Instances _
            .Where(Function(x) TypeOf (x) Is SPSearchServiceInstance).Single(), 
            SPSearchServiceInstance)
        Dim lv As Object = reflected("LogViewer").GetConstructor({GetType(SPSearchServiceInstance)}) _
            .Invoke({siSearchServiceInstance})

        Dim dt As DataTable
        Dim nextStart As Integer = 1

        If arguments.exportType = "SUMMARY" Then
            outTextWriter.WriteLine("Crawl Log Summary")
            outTextWriter.WriteLine("=============================")
            outTextWriter.WriteLine()

            Console.WriteLine("MaxDaysCrawlLogged:" & lv.MaxDaysCrawlLogged.ToString())
            Console.WriteLine("Generating Status Messages Summary")

            Dim dtStatusMessages As DataTable
            dtStatusMessages = lv.GetAllStatusMessages()
            dtStatusMessages.TableName = "StatusMessages"

            Console.WriteLine("Generating Crawl Summary By Host")
            Do While nextStart > 0
                dt = DirectCast(lv.GetCrawlSummaryByHost(
                    System.Enum.Parse(reflected("CrawlLogSummaryProperty"), "ErrorCount") And _
                    System.Enum.Parse(reflected("CrawlLogSummaryProperty"), "HostName") And _
                    System.Enum.Parse(reflected("CrawlLogSummaryProperty"), "SuccessCount") And _
                    System.Enum.Parse(reflected("CrawlLogSummaryProperty"), "TotalCount") And _
                    System.Enum.Parse(reflected("CrawlLogSummaryProperty"), "WarningCount"), _
                    Search.Query.SortDirection.Ascending, _
                    arguments.startPos, _
                    arguments.maxCount, _
                    nextStart), DataTable)
                dt.TableName = "CrawlSummaryByHost"
                ' text output header
                outTextWriter.WriteLine()
                For Each col As DataColumn In dt.Columns
                    outTextWriter.Write(col.Caption() & vbTab)
                Next
                outTextWriter.WriteLine()
                For Each rwHostName As DataRow In dt.Rows
                    ' text output hostname summary row.
                    outTextWriter.WriteLine()
                    For Each col As DataColumn In rwHostName.Table.Columns
                        outTextWriter.Write(rwHostName(col).ToString() & vbTab)
                    Next
                    outTextWriter.WriteLine()
                    ' compute hostname summary by messageid
                    Dim dtErrorByHost As DataTable
                    Dim clf As Object = reflected("CrawlLogFilters").GetConstructor({}).Invoke({})
                    clf.AddFilter(arguments.startDT, arguments.endDT)
                    clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "StartAt"), arguments.startPos)
                    clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "TotalEntries"), arguments.maxCount)
                    clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "HostName"),
                                  System.Enum.Parse(reflected("StringFilterOperator"), "Equal"), rwHostName("HostName"))
                    ' query to get grand total
                    dtErrorByHost = DirectCast(lv.GetCurrentCrawlLogData(clf, 1), DataTable)
                    dtErrorByHost.TableName = "CurrentCrawlLogData"
                    outTextWriter.WriteLine(dtErrorByHost.Rows.Count.ToString().PadLeft(10) & " : Grand total for " & rwHostName("HostName") & " from:" & arguments.startDT.ToUniversalTime & " to:" & arguments.endDT.ToUniversalTime)

                    Dim clfSubTotal As Object = reflected("CrawlLogFilters").GetConstructor({}).Invoke({})
                    For Each rwErr As DataRow In dtStatusMessages.Rows
                        clfSubTotal = clf
                        ' now, subtotal by errorid's
                        clfSubTotal.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "MessageId"), rwErr("ErrorId"))
                        dtErrorByHost = DirectCast(lv.GetCurrentCrawlLogData(clfSubTotal, 1), DataTable)
                        If dtErrorByHost.Rows.Count <> 0 Then
                            dtErrorByHost.TableName = "CurrentCrawlLogData"
                            outTextWriter.WriteLine(dtErrorByHost.Rows.Count.ToString.PadLeft(10) & " :" & rwErr("ErrorId").ToString.PadLeft(5) & ":" & rwErr("ErrorMsg"))
                        End If
                    Next ' error
                    outTextWriter.WriteLine()
                Next ' hostname
            Loop

        ElseIf arguments.exportType = "DETAILS" Then
            Console.WriteLine("Exporting Current Crawl Log Data")
            Dim clf As Object = reflected("CrawlLogFilters").GetConstructor({}).Invoke({})
            clf.AddFilter(arguments.startDT, arguments.endDT)
            clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "StartAt"), arguments.startPos)
            clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "TotalEntries"), arguments.maxCount)
            If Not arguments.catalog Is Nothing Then
                clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "CatalogType"), System.Enum.Parse(reflected("StringFilterOperator"), "Equal"), arguments.catalog)
            End If
            If arguments.messageId > Integer.MinValue Then
                clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "MessageId"), arguments.messageId)
            End If
            If arguments.contentSourceId > Integer.MinValue Then
                clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "ContentSourceId"), arguments.contentSourceId)
            End If
            If Not arguments.URLPattern Is Nothing Then
                clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "Url"), System.Enum.Parse(reflected("StringFilterOperator"), "Contains"), arguments.URLPattern)
            End If
            If Not arguments.HostName Is Nothing Then
                clf.AddFilter(System.Enum.Parse(reflected("CrawlLogFilterProperty"), "HostName"), System.Enum.Parse(reflected("StringFilterOperator"), "Equal"), arguments.HostName)
            End If
            ' message type filter options are mutually exclusive, last one set wins.
            If arguments.messageType.Contains("s") Then
                clf.AddFilter(System.Enum.Parse(reflected("MessageType"), "Warning"))
            End If
            If arguments.messageType.Contains("w") Then
                clf.AddFilter(System.Enum.Parse(reflected("MessageType"), "Warning"))
            End If
            If arguments.messageType.Contains("e") Then
                clf.AddFilter(System.Enum.Parse(reflected("MessageType"), "Error"))
            End If
            Dim writeHeader As Boolean = True
            Do While nextStart > 0
                dt = DirectCast(lv.GetCurrentCrawlLogData(clf, nextStart), DataTable)
                dt.TableName = "CurrentCrawlLogData"
                If writeHeader Then
                    outTextWriter.Write("LastTouchStart" & vbTab)
                    outTextWriter.Write("ContentSourceId" & vbTab)
                    outTextWriter.Write("ErrorLevel" & vbTab)
                    outTextWriter.Write("HResult" & vbTab)
                    outTextWriter.Write("DisplayUrl" & vbTab)
                    outTextWriter.Write("ErrorMsg" & vbTab)
                    outTextWriter.Write("ErrorDesc" & vbTab)
                    ' I didn't like the default column order, sorry.
                    'For Each col As DataColumn In dt.Columns
                    '    outTextWriter.Write(col.ColumnName & vbTab)
                    'Next
                    outTextWriter.WriteLine()
                    writeHeader = False
                End If
                Dim dt2 As DataTable
                For Each row As DataRow In dt.Rows
                    outTextWriter.Write(row("LastTouchStart").ToString() & vbTab)
                    outTextWriter.Write(row("ContentSourceId").ToString() & vbTab)
                    outTextWriter.Write(row("ErrorLevel").ToString() & vbTab)
                    outTextWriter.Write(row("HResult").ToString() & vbTab)
                    outTextWriter.Write(row("DisplayUrl").ToString() & vbTab)
                    outTextWriter.Write(row("ErrorMsg").ToString().Replace(vbCrLf, " ").Replace(vbCr, " ") & vbTab)
                    outTextWriter.Write(row("ErrorDesc").ToString().Replace(vbCrLf, " ").Replace(vbCr, " ") & vbTab)
                    ' I didn't like the default column order, sorry.
                    'For Each col As DataColumn In dt.Columns
                    '    outTextWriter.Write(row(col.Ordinal).ToString().Replace(vbCrLf, " ").Replace(vbCr, " ") & vbTab)
                    'Next
                    If arguments.dumpHistory Then
                        'Console.WriteLine("Exporting Crawl Log Historical Data")
                        ' guessing that only non-zero ErrorLevel items have pastcrawllogdata. a.k.a. "history"
                        'If row("ErrorLevel") <> 0 Then
                        dt2 = DirectCast(lv.GetPastCrawlLogData(New Uri(DirectCast(row("DisplayUrl"), String))), DataTable)
                        If dt2.Rows.Count > 0 Then
                            dt2.TableName = "PastCrawlLogData"
                            For Each row2 As DataRow In dt2.Rows
                                For Each col2 As DataColumn In dt2.Columns
                                    outTextWriter.Write(row2(col2.Ordinal).ToString().Replace(vbCrLf, " ").Replace(vbCr, " ") & vbTab)
                                Next
                            Next
                        End If
                    End If
                    outTextWriter.WriteLine()
                Next
            Loop
        ElseIf arguments.exportType = "CSREPORT" Then
            outTextWriter.WriteLine("Content Source Summary Report")
            outTextWriter.WriteLine("=============================")
            outTextWriter.WriteLine()
            Dim gathererApp As Object = GetType(SPSearchServiceInstance) _
                .GetProperty("GathererApplication", BindingFlags.Instance Or BindingFlags.NonPublic) _
                .GetValue(siSearchServiceInstance, Nothing)
            Dim gathererProj As Object = gathererApp.GetType() _
                .GetProperty("PrimaryGathererProject", BindingFlags.Instance Or BindingFlags.NonPublic) _
                .GetValue(gathererApp, Nothing)
            Dim contentSources As Object = gathererProj.GetType() _
                .GetProperty("ContentSources", BindingFlags.Instance Or BindingFlags.NonPublic) _
                    .GetValue(gathererProj, Nothing)
            For Each sspContentSource As Object In DirectCast(contentSources, IEnumerable)
                For Each prop As String In {"ContentSourceID", "ContentSourceName",
                                            "ContentSourceType",
                                            "CrawlStarted", "CrawlCompleted",
                                            "CrawlSuccesses", "CrawlWarnings", "CrawlErrors", "CrawlLevelHighErrors",
                                            "EnumerationDepth",
                                            "CrawlDeletes",
                                            "CrawlPriority",
                                            "FollowDirectories", "FullCrawlTrigger",
                                            "HostDepth", "IncCrawlTrigger",
                                            "Metadata", "ReadOnly",
                                            "Status", "WssCrawlStyle"}
                    Dim value As Object
                    Try
                        value = reflected("IContentSource").GetProperty(prop).GetValue(sspContentSource, Nothing)
                        If value Is Nothing Then value = "Nothing"
                    Catch ex As Exception
                        value = ex.ToString()
                    End Try
                    outTextWriter.WriteLine(prop & " : " & value.ToString())
                Next
                outTextWriter.WriteLine(" Start Addresses:")
                For Each sspSA As Object In DirectCast(sspContentSource.StartAddresses, IEnumerable)
                    outTextWriter.WriteLine(" " & sspSA.URL)
                Next
                outTextWriter.WriteLine()
            Next

        ElseIf arguments.exportType = "PURGELOG" Then
                outTextWriter.Write("Purging Crawl Log...")
                lv.PurgeCrawlLog()
                outTextWriter.WriteLine("Done")
        End If
        outTextWriter.Flush()
        Return 0
    End Function

    Public Function ParseArgs(ByVal keyValues As System.Collections.Specialized.StringDictionary) As Boolean
        Dim argsAreGood As Boolean = True
        If keyValues.ContainsKey("t") Then
            ' report type
            If keyValues("t").ToLowerInvariant = "d" Then
                arguments.exportType = "DETAILS"
            ElseIf keyValues("t").ToLowerInvariant = "s" Then
                arguments.exportType = "SUMMARY"
            ElseIf keyValues("t").ToLowerInvariant = "p" Then
                arguments.exportType = "PURGELOG"
            ElseIf keyValues("t").ToLowerInvariant = "c" Then
                arguments.exportType = "CSREPORT"
            Else
                argsAreGood = False
            End If
        Else
            argsAreGood = False
        End If
        If keyValues.ContainsKey("site") Then arguments.site = keyValues("site")

        If keyValues.ContainsKey("outfile") Then arguments.outfile = keyValues("outfile")

        arguments.dumpHistory = keyValues.ContainsKey("history")

        If keyValues.ContainsKey("s") Then arguments.startDT = Date.Parse(keyValues("s"))

        If keyValues.ContainsKey("e") Then arguments.endDT = Date.Parse(keyValues("e"))

        If keyValues.ContainsKey("from") Then arguments.startPos = Integer.Parse(keyValues("from"))

        If keyValues.ContainsKey("thru") Then arguments.maxCount = Integer.Parse(keyValues("thru"))

        If keyValues.ContainsKey("cat") Then arguments.catalog = keyValues("cat")

        If keyValues.ContainsKey("csid") Then arguments.contentSourceId = Integer.Parse(keyValues("csid"))

        If keyValues.ContainsKey("msgid") Then arguments.messageId = Integer.Parse(keyValues("msgid"))

        If keyValues.ContainsKey("u") Then arguments.URLPattern = keyValues("u")

        If keyValues.ContainsKey("mt") Then arguments.messageType = keyValues("mt").ToLowerInvariant()

        If keyValues.ContainsKey("hostname") Then arguments.HostName = keyValues("hostname")

        ' check for minimum requirements
        If arguments.exportType Is Nothing _
           Then
            ' some required item is missing
            argsAreGood = False
        End If

        Return argsAreGood
    End Function

End Class
